package test3.scuec.day2.crawl;
import java.io.*;

import java.net.HttpURLConnection;

import java.net.URL;





import java.io.File;

import java.io.BufferedReader;

import java.io.FileOutputStream;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStreamWriter;

import java.io.PrintWriter;

import java.net.URL;

import java.net.URLConnection;

import java.util.ArrayList;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import java.util.Hashtable;



/**

 * Created by zt on 2016/6/27.

 */

public class Utils {



    private static void setRequestPro(HttpURLConnection urlConn) {



        //User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36

        urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");

        urlConn.setRequestProperty("Accept",

                "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-powerpoint, application/vnd.ms-excel, application/msword, */*");

        urlConn.setRequestProperty("Accept-Language", "zh-cn");

        urlConn.setRequestProperty("Accept-Encoding", "gzip");

        urlConn.setRequestProperty("Content-type", "text/html");

        urlConn.setRequestProperty("Connection", "close"); //keep-Alive，



        urlConn.setUseCaches(false);

        urlConn.setConnectTimeout(10 * 1000);

        urlConn.setReadTimeout(10 * 1000);

        urlConn.setDoOutput(true);

        urlConn.setDoInput(true);

    }





    public static String GetWebContent(String urlString, final String charset, int timeout) throws IOException {

        if (urlString == null || urlString.length() == 0) {

            return null;

        }

        urlString = (urlString.startsWith("http://") || urlString.startsWith("https://")) ? urlString : ("http://" + urlString);

        URL url = new URL(urlString);

        HttpURLConnection conn = (HttpURLConnection) url.openConnection();

        conn.setDoOutput(true);

        conn.setRequestProperty("Pragma", "no-cache");

        conn.setRequestProperty("Cache-Control", "no-cache");



        int temp = Integer.parseInt(Math.round(Math.random() * 7) + "");

        String ua = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";

        conn.setRequestProperty("User-Agent", ua);//UserAgent[temp]);

        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");//只接受text/html类型，当然也可以接受图片,pdf,*/*任意，就是tomcat/conf/web里面定义那些

        conn.setConnectTimeout(timeout);

        try {

            System.out.println("conn.getResponseCode():" + conn.getResponseCode());

            System.out.println("conn.getResponseMessage():" + conn.getResponseMessage());



            if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {

                return null;

            }

        } catch (Exception e) {

            try {

                e.printStackTrace();

            } catch (Exception e2) {

                e2.printStackTrace();

            }

            return null;

        }



        InputStream input = conn.getInputStream();

       //  write2String(input,charset);

      write2File(input,"d:\\temp\\download","news163.txt",charset);

        if (conn != null) {

            conn.disconnect();

        }

        return null ;

    }



  /*  private static String[] UserAgent = {

            "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",

            "Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",

            "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121",

            "UCWEB7.0.2.37/28/998",

            "NOKIA5700/UCWEB7.0.2.37/28/977",

            "Openwave/UCWEB7.0.2.37/28/978",

            "Openwave/UCWEB7.0.2.37/28/978",

            "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989"

    };*/



    public  static String getDomain() {

        String reg = "(?<=http\\://[a-zA-Z0-9]{0,100}[.]{0,1})[^.\\s]*?\\.(com|cn|net|org|biz|info|cc|tv)";

        Pattern p = Pattern.compile(reg, Pattern.CASE_INSENSITIVE);

        Matcher m = p.matcher("http://news.163.com/");

        boolean blnp = m.find();

        if (blnp == true) {

            return m.group(0);

        }

        return null;

    }

    

    public static boolean isNotEmpty(Object o) {

        if (null == o) {

            return false;

        }

        if (o.toString().equals("")) {

            return false;

        }

        return true;

    }



    public static void write2File(InputStream in, String path, String fileName,String charset) {

        File fp = new File(path);

        if(!fp.exists()){

            fp.mkdirs();

        }

        try {



            String filePath =fp +"\\"+ fileName;

            PrintWriter pw = null;

            FileOutputStream fos = new FileOutputStream(filePath);

            OutputStreamWriter writer = new OutputStreamWriter(fos,"utf-8");

            pw = new PrintWriter(writer);

            BufferedReader bReader = new BufferedReader(new InputStreamReader(in,charset));

            StringBuffer sb = new StringBuffer();

            String rLine = null;

            while ((rLine = bReader.readLine()) != null) {

                    pw.println(rLine);

                    pw.flush();

            }

            in.close();

            pw.close();

        }catch (Exception e){

            e.printStackTrace();

        }



    }



    public static void write2String(InputStream input, String charset){

        try {

            BufferedReader reader = new BufferedReader(new InputStreamReader(input,

                    charset));

            String line = null;

            StringBuffer sb = new StringBuffer();

            while ((line = reader.readLine()) != null) {

                sb.append(line).append("\r\n");

            }

            if (reader != null) {

                reader.close();

            }

            System.out.println(sb.toString());/////////////////////打印网页内容

        }catch (Exception e){

            e.printStackTrace();

        }

    }



}

